{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "import os\n",
    "import math\n",
    "import pandas as pd\n",
    "from glob import iglob\n",
    "import numpy as np\n",
    "import time\n",
    "from keras.utils import plot_model\n",
    "from keras.models import load_model\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Input, Dense\n",
    "from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping\n",
    "from keras.optimizers import SGD\n",
    "from sklearn.metrics import recall_score, accuracy_score, precision_score, confusion_matrix\n",
    "from utils import plot_confusion_matrix\n",
    "import lime\n",
    "import lime.lime_tabular"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading training data...\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading training data...\")\n",
    "df = pd.concat((pd.read_csv(f) for f in iglob('../data/**/benign_traffic.csv', recursive=True)), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_opt, x_test = np.split(df.sample(frac=1, random_state=17), [int(1/3*len(df)), int(2/3*len(df))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mirai = pd.concat((pd.read_csv(f) for f in iglob('../data/**/mirai_attacks/*.csv', recursive=True)), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gafgyt = pd.concat((pd.read_csv(f) for f in iglob('../data/**/gafgyt_attacks/*.csv', recursive=True)), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_attack = df_mirai.append(df_gafgyt)\n",
    "df_attack['class'] = 'attack'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fish = x_train.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fish['class'] = 'benign'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fish = df_fish.append(df_attack.sample(n=df_fish.shape[0], random_state=17))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_n = 2000\n",
    "atk = df_attack.sample(n=plot_n, random_state=76)\n",
    "nrm = x_train.sample(n=plot_n, random_state=42)\n",
    "\n",
    "fig, ax1 = plt.subplots()\n",
    "\n",
    "ax1.scatter(nrm['MI_dir_L0.1_weight'],nrm['MI_dir_L1_weight'],10,c='blue', label='normal',alpha=0.5)\n",
    "ax1.scatter(atk['MI_dir_L0.1_weight'],atk['MI_dir_L1_weight'],10,c='red',label='attack',alpha=0.5)\n",
    "\n",
    "plt.xlabel('MI_dir_L0.1_weight')\n",
    "plt.ylabel('MI_dir_L1_weight')\n",
    "\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ['benign', 'attack']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'feature': 'MI_dir_L0.1_weight', 'score': 3.3424566344942805},\n",
       " {'feature': 'H_L0.1_weight', 'score': 3.34245638990826},\n",
       " {'feature': 'MI_dir_L1_weight', 'score': 3.1793815222225743},\n",
       " {'feature': 'H_L1_weight', 'score': 3.1793812343865087},\n",
       " {'feature': 'MI_dir_L3_weight', 'score': 3.0088024641835225}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scored = []\n",
    "indices = {}\n",
    "shps = {}\n",
    "for cl in classes:\n",
    "    indices[cl] = df_fish['class'] == cl\n",
    "    shps[cl] =  df_fish[indices[cl]].shape[0]\n",
    "        \n",
    "for col in df_fish.columns:\n",
    "    if col == 'class':\n",
    "        continue\n",
    "    num = 0\n",
    "    den = 0\n",
    "    m = df_fish[col].mean()\n",
    "    \n",
    "    for cl in classes:\n",
    "        num += (shps[cl] / df_fish.shape[0]) * (m - df_fish[indices[cl]][col].mean())**2\n",
    "        den += (shps[cl] / df_fish.shape[0]) * df_fish[indices[cl]][col].var()\n",
    "    score = {'feature': col, 'score': num / den}\n",
    "    scored.append(score)\n",
    "    #print(score)\n",
    "scored.sort(key=lambda x: x['score'], reverse=True)\n",
    "scored[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('anomaly_scores.csv', 'w+') as file:\n",
    "    lines = ['Feature,Score\\n']\n",
    "    for s in scored:\n",
    "        lines.append(s['feature'] + ',' + \"{0:.2f}\".format(s['score']) + '\\n')\n",
    "    file.writelines(lines)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(input_dim):\n",
    "    inp = Input(shape=(input_dim,))\n",
    "    encoder = Dense(int(math.ceil(0.75 * input_dim)), activation=\"tanh\")(inp)\n",
    "    encoder = Dense(int(math.ceil(0.5 * input_dim)), activation=\"tanh\")(encoder)\n",
    "    encoder = Dense(int(math.ceil(0.25 * input_dim)), activation=\"tanh\")(encoder)\n",
    "    decoder = Dense(int(math.ceil(0.5 * input_dim)), activation=\"tanh\")(encoder)\n",
    "    decoder = Dense(int(math.ceil(0.75 * input_dim)), activation=\"tanh\")(decoder)\n",
    "    decoder = Dense(input_dim)(decoder)\n",
    "    return Model(inp, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AnomalyModel:\n",
    "    def __init__(self, model, threshold, scaler):\n",
    "        self.model = model\n",
    "        self.threshold = threshold\n",
    "        self.scaler = scaler\n",
    "\n",
    "    def predict(self, x):\n",
    "        x_pred = self.model.predict(x)\n",
    "        mse = np.mean(np.power(x - x_pred, 2), axis=1)\n",
    "        y_pred = mse > self.threshold\n",
    "        return y_pred.astype(int)\n",
    "\n",
    "    def scale_predict_classes(self, x):\n",
    "        x = self.scaler.transform(x)\n",
    "        y_pred = self.predict(x)\n",
    "        classes_arr = []\n",
    "        for e in y_pred:\n",
    "            el = [0,0]\n",
    "            el[e] = 1\n",
    "            classes_arr.append(el)\n",
    "\n",
    "        return np.array(classes_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 185310 samples, validate on 185311 samples\n",
      "Epoch 1/100\n",
      "185310/185310 [==============================] - 24s 128us/step - loss: 0.1436 - val_loss: 0.0767\n",
      "Epoch 2/100\n",
      "185310/185310 [==============================] - 25s 136us/step - loss: 0.0654 - val_loss: 0.0527\n",
      "Epoch 3/100\n",
      "185310/185310 [==============================] - 26s 138us/step - loss: 0.0514 - val_loss: 0.0453\n",
      "Epoch 4/100\n",
      "185310/185310 [==============================] - 25s 133us/step - loss: 0.0457 - val_loss: 0.0409\n",
      "Epoch 5/100\n",
      "185310/185310 [==============================] - 25s 134us/step - loss: 0.0425 - val_loss: 0.0405\n",
      "Epoch 6/100\n",
      "185310/185310 [==============================] - 26s 139us/step - loss: 0.0403 - val_loss: 0.0372\n",
      "Epoch 7/100\n",
      "185310/185310 [==============================] - 25s 136us/step - loss: 0.0377 - val_loss: 0.0369\n",
      "Epoch 8/100\n",
      "185310/185310 [==============================] - 25s 137us/step - loss: 0.0362 - val_loss: 0.0339\n",
      "Epoch 9/100\n",
      "185310/185310 [==============================] - 26s 138us/step - loss: 0.0352 - val_loss: 0.0318\n",
      "Epoch 10/100\n",
      "185310/185310 [==============================] - 25s 135us/step - loss: 0.0338 - val_loss: 0.0375\n",
      "Epoch 11/100\n",
      "185310/185310 [==============================] - 25s 132us/step - loss: 0.0340 - val_loss: 0.0318\n",
      "Epoch 12/100\n",
      "185310/185310 [==============================] - 25s 133us/step - loss: 0.0325 - val_loss: 0.0319\n",
      "Epoch 13/100\n",
      "185310/185310 [==============================] - 24s 130us/step - loss: 0.0322 - val_loss: 0.0308\n",
      "Epoch 14/100\n",
      "185310/185310 [==============================] - 24s 130us/step - loss: 0.0319 - val_loss: 0.0333\n",
      "Epoch 15/100\n",
      "185310/185310 [==============================] - 24s 130us/step - loss: 0.0305 - val_loss: 0.0311\n",
      "Epoch 16/100\n",
      "185310/185310 [==============================] - 25s 134us/step - loss: 0.0298 - val_loss: 0.0301\n",
      "Epoch 17/100\n",
      "185310/185310 [==============================] - 24s 131us/step - loss: 0.0302 - val_loss: 0.0298\n",
      "Epoch 18/100\n",
      "185310/185310 [==============================] - 24s 131us/step - loss: 0.0290 - val_loss: 0.0300\n",
      "Epoch 19/100\n",
      "185310/185310 [==============================] - 25s 134us/step - loss: 0.0292 - val_loss: 0.0321\n",
      "Epoch 20/100\n",
      "185310/185310 [==============================] - 25s 133us/step - loss: 0.0284 - val_loss: 0.0278\n",
      "Epoch 21/100\n",
      "185310/185310 [==============================] - 25s 134us/step - loss: 0.0287 - val_loss: 0.0272\n",
      "Epoch 22/100\n",
      "185310/185310 [==============================] - 25s 136us/step - loss: 0.0277 - val_loss: 0.0285\n",
      "Epoch 23/100\n",
      "185310/185310 [==============================] - 25s 137us/step - loss: 0.0280 - val_loss: 0.0279\n",
      "Epoch 24/100\n",
      "185310/185310 [==============================] - 25s 135us/step - loss: 0.0280 - val_loss: 0.0258\n",
      "Epoch 25/100\n",
      "185310/185310 [==============================] - 25s 133us/step - loss: 0.0278 - val_loss: 0.0300\n",
      "Epoch 26/100\n",
      "185310/185310 [==============================] - 25s 133us/step - loss: 0.0274 - val_loss: 0.0291\n",
      "Epoch 27/100\n",
      "185310/185310 [==============================] - 25s 136us/step - loss: 0.0270 - val_loss: 0.0298\n",
      "Epoch 28/100\n",
      "185310/185310 [==============================] - 25s 134us/step - loss: 0.0268 - val_loss: 0.0265\n",
      "Epoch 29/100\n",
      "185310/185310 [==============================] - 24s 132us/step - loss: 0.0273 - val_loss: 0.0260\n",
      "time\n",
      "720.8051059246063\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 115)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 87)                10092     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 58)                5104      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 29)                1711      \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 58)                1740      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 87)                5133      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 115)               10120     \n",
      "=================================================================\n",
      "Total params: 33,900\n",
      "Trainable params: 33,900\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Selecting n------------------\n",
      "For n 1\n",
      "Accuracy\n",
      "0.9909152721640917\n",
      "Precision\n",
      "0.9822263450834879\n",
      "CM\n",
      "[[181958   3353]\n",
      " [    14 185297]]\n",
      "For n 2\n",
      "Accuracy\n",
      "0.9947763489485244\n",
      "Precision\n",
      "0.989739179668513\n",
      "CM\n",
      "[[183390   1921]\n",
      " [    15 185296]]\n",
      "For n 3\n",
      "Accuracy\n",
      "0.9964627032394191\n",
      "Precision\n",
      "0.9930649345081142\n",
      "CM\n",
      "[[184017   1294]\n",
      " [    17 185294]]\n",
      "For n 4\n",
      "Accuracy\n",
      "0.997455628645898\n",
      "Precision\n",
      "0.9950539713227001\n",
      "CM\n",
      "[[184390    921]\n",
      " [    22 185289]]\n",
      "For n 5\n",
      "Accuracy\n",
      "0.9980033565195806\n",
      "Precision\n",
      "0.9961666048377125\n",
      "CM\n",
      "[[184598    713]\n",
      " [    27 185284]]\n"
     ]
    }
   ],
   "source": [
    "for top_n_features in [115]:\n",
    "    fs = [it['feature'] for it in scored[:top_n_features]]\n",
    "    X_train = x_train[fs]\n",
    "    X_opt = x_opt[fs]\n",
    "    X_test = x_test[fs]\n",
    "    \n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_opt = scaler.transform(X_opt)\n",
    "    \n",
    "    model = create_model(top_n_features)\n",
    "    model.compile(loss=\"mean_squared_error\",\n",
    "                    optimizer=\"adam\")\n",
    "    cp = ModelCheckpoint(filepath=f\"anomaly/anomaly{top_n_features}.h5\",\n",
    "                                 monitor='val_loss',\n",
    "                               save_best_only=True,\n",
    "                               verbose=0)\n",
    "\n",
    "    es = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n",
    "    start = time.time()\n",
    "    epochs = 100\n",
    "    history = model.fit(X_train, X_train,\n",
    "                    epochs=epochs,\n",
    "                    validation_data=(X_opt, X_opt),\n",
    "                    verbose=1,\n",
    "                    callbacks=[cp, es])\n",
    "    \n",
    "    end = time.time()\n",
    "    print('time')\n",
    "    print(end - start)\n",
    "    print(model.summary())\n",
    "    \n",
    "    x_opt_predictions = model.predict(X_opt)\n",
    "    mse = np.mean(np.power(X_opt - x_opt_predictions, 2), axis=1)\n",
    "    mean = mse.mean()\n",
    "    std = mse.std()\n",
    "    \n",
    "    \n",
    "    df_benign = pd.DataFrame(x_opt[fs], columns=fs)\n",
    "    df_benign['malicious'] = 0\n",
    "    df_malicious = df_attack.sample(n=df_benign.shape[0], random_state=39)[fs]\n",
    "    df_malicious['malicious'] = 1\n",
    "    df2 = df_benign.append(df_malicious)\n",
    "    X_opt = df2.drop(columns=['malicious']).values\n",
    "    X_opt_scaled = scaler.transform(X_opt)\n",
    "    \n",
    "    Y_opt = df2['malicious']\n",
    "    best_acc = 0\n",
    "    best_n = 0\n",
    "    print('Selecting n------------------')\n",
    "    for n in range(1,11):\n",
    "        tr = mean + n * std\n",
    "        m = AnomalyModel(model , tr, scaler)\n",
    "        Y_pred = m.predict(X_opt_scaled)\n",
    "        print(f'For n {n}')\n",
    "        print('Accuracy')\n",
    "        acc = accuracy_score(Y_opt, Y_pred)\n",
    "        if acc > best_acc:\n",
    "            best_acc = acc\n",
    "            best_n = n\n",
    "        print(acc)\n",
    "        print('Precision')\n",
    "        print(precision_score(Y_opt, Y_pred))\n",
    "        print('CM')\n",
    "        print(confusion_matrix(Y_opt, Y_pred))\n",
    "        #accs.append({'acc': acc, 'n': top_n_features, 'cm': confusion_matrix(Y_test, Y_pred)})\n",
    "        \n",
    "    df_benign = pd.DataFrame(X_test, columns=fs)\n",
    "    df_benign['malicious'] = 0\n",
    "    df_malicious = df_attack.sample(n=df_benign.shape[0], random_state=17)[fs]\n",
    "    df_malicious['malicious'] = 1\n",
    "    df2 = df_benign.append(df_malicious)\n",
    "    X_test = df2.drop(columns=['malicious']).values\n",
    "    X_test_scaled = scaler.transform(X_test)\n",
    "    Y_test = df2['malicious']\n",
    "    \n",
    "    tr = mean + best_n * std\n",
    "    m = AnomalyModel(model , tr, scaler)\n",
    "    Y_pred = m.predict(X_test_scaled)\n",
    "    print('Test-----------------')\n",
    "    print(f'best n {best_n}')\n",
    "    print('Accuracy')\n",
    "    acc = accuracy_score(Y_test, Y_pred)\n",
    "    print(acc)\n",
    "    print('Precision')\n",
    "    print(precision_score(Y_test, Y_pred))\n",
    "    print('CM')\n",
    "    cm = confusion_matrix(Y_test, Y_pred)\n",
    "    print(cm)   \n",
    "#print(accs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XGXZ//HPNZNl0mZrk7RNutCFsrRlKwHEoggIskmhoILwE3DhccFHX7g8+PP5CQ9uqIgiooICghsKAlblsaK4IWspLdAFmpbSpk3bNKVNtywzc/3+OCftNM0yaTOZNPN9v17zmjPn3GfONZPJXHPf97nPbe6OiIhITyLZDkBERAY/JQsREemVkoWIiPRKyUJERHqlZCEiIr1SshARkV4pWYgcADObaGZuZnlplL3KzJ480OcRyQYlC8kZZrbKzNrMrLLT+hfDL+qJ2YlMZPBTspBc8zpwWccDMzsKGJa9cEQODkoWkmt+Bnwg5fGVwP2pBcyszMzuN7NGM3vDzP7bzCLhtqiZ3WJmm8xsJXBeF/vebWYNZrbWzL5iZtG+BmlmNWY218w2m1mdmX0kZduJZjbfzJrNbIOZ3Rquj5nZz82sycy2mNnzZja6r8cW6YqSheSaZ4BSMzsy/BK/FPh5pzK3A2XAZOBUguRydbjtI8D5wHFALXBJp31/CsSBQ8MyZwEf3o84HwDqgZrwGF8zs9PDbbcBt7l7KTAF+E24/sow7vFABfBRYNd+HFtkH0oWkos6ahdnAkuBtR0bUhLIF9x9m7uvAr4N/J+wyHuB77r7GnffDHw9Zd/RwLnAp919h7tvBL4TPl/azGw8MAv4L3dvcfeFwE/YUyNqBw41s0p33+7uz6SsrwAOdfeEu7/g7s19ObZId5QsJBf9DHg/cBWdmqCASiAfeCNl3RvA2HC5BljTaVuHQ8J9G8JmoC3AncCoPsZXA2x2923dxPAh4DBgWdjUdH7K65oHPGBm68zsm2aW38dji3RJyUJyjru/QdDRfS7wcKfNmwh+oR+Ssm4Ce2ofDQTNPKnbOqwBWoFKdy8Pb6XuPr2PIa4DRppZSVcxuPtyd7+MIAl9A3jIzIa7e7u7/4+7TwPeStBc9gFE+oGSheSqDwGnu/uO1JXuniDoA/iqmZWY2SHAdezp1/gN8J9mNs7MRgDXp+zbAPwZ+LaZlZpZxMymmNmpfQnM3dcATwFfDzutjw7j/TmAmV1hZlXungS2hLslzew0MzsqbEprJkh6yb4cW6Q7ShaSk9x9hbvP72bzJ4EdwErgSeCXwD3hth8TNPUsAhawb83kA0ABsAR4E3gIqN6PEC8DJhLUMh4BbnD3v4TbzgYWm9l2gs7uS919FzAmPF4zQV/MPwiapkQOmGnyIxER6Y1qFiIi0islCxER6ZWShYiI9ErJQkREejVkLodcWVnpEydOzHYYIiIHlRdeeGGTu1f1Vm7IJIuJEycyf353Z0KKiEhXzOyN3kupGUpERNKgZCEiIr1SshARkV4NmT6LrrS3t1NfX09LS0u2QxkwsViMcePGkZ+vi42KSP8Z0smivr6ekpISJk6ciJllO5yMc3eampqor69n0qRJ2Q5HRIaQId0M1dLSQkVFRU4kCgAzo6KiIqdqUiIyMIZ0sgByJlF0yLXXKyIDY8gni97Ek0k2NLewsy2e7VBERAatjCYLMzvbzF41szozu76L7W83swVmFjezzhPfE04gU29m389knBuaW9jRmuj3521qauLYY4/l2GOPZcyYMYwdO3b347a2trSe4+qrr+bVV1/t99hERPoiYx3c4WxddwBnAvXA82Y2192XpBRbTTAP8me7eZovA//MVIwAUTMiZrQn+n9CsYqKChYuXAjAjTfeSHFxMZ/97N4v1d1xdyKRrvP2vffe2+9xiYj0VSZrFicCde6+0t3bgAeA2akF3H2Vu79EF1M/mtnxwGiCaSozxszIj0Yykiy6U1dXx7Rp07j88suZPn06DQ0NXHPNNdTW1jJ9+nRuuumm3WVPOeUUFi5cSDwep7y8nOuvv55jjjmGk08+mY0bNw5YzCKS2zJ56uxYggnsO9QDJ6Wzo5lFgG8DVwDv7KHcNcA1ABMmTOjxOf/n94tZsq65y20t7QkcKMqPphPebtNqSrnh3dP7tE+HZcuWcf/991NbWwvAzTffzMiRI4nH45x22mlccsklTJs2ba99tm7dyqmnnsrNN9/Mddddxz333MP11+/Tuici0u8Gawf3x4HH3L2+p0Lufpe717p7bVVVrxdN7JaZMdCzy06ZMmV3ogD41a9+xcyZM5k5cyZLly5lyZIl++xTVFTEOeecA8Dxxx/PqlWrBipcEclxmaxZrAXGpzweF65Lx8nA28zs40AxUGBm2919v39G91QD2NDcwobmFmaMLSMyQKeeDh8+fPfy8uXLue2223juuecoLy/niiuu6HKsREFBwe7laDRKPK4zuERkYGSyZvE8MNXMJplZAXApMDedHd39cnef4O4TCTq/7z+QRNGb/GiQIOID2G+Rqrm5mZKSEkpLS2loaGDevHlZiUNEpDsZq1m4e9zMrgXmAVHgHndfbGY3AfPdfa6ZnQA8AowA3m1m/+Pu+9cJcADyo0HObE84BVm4AMrMmTOZNm0aRxxxBIcccgizZs0a+CBERHpgPtCN9RlSW1vrnSc/Wrp0KUceeWSv+7a0J3htwzYmjBxG+bCCXssPdum+bhERM3vB3Wt7KzdYO7gH1J6aRXaaoUREBjslCyAaMaJmtCeGRi1LRKS/KVmE8vMGdmCeiMjBRMkilB+N0KZkISLSJSWLUH7UaI+rGUpEpCtKFqH8aIR4MklyiJwdJiLSn5QsQh1nRGViYF59fT2zZ89m6tSpTJkyhU996lM9XqJ8y5Yt/OAHP+j3OERE9peSRahjFHdbP58R5e7MmTOHCy+8kOXLl/Paa6+xfft2vvjFL3a7j5KFiAw2WRivPDhlaqzFE088QSwW4+qrrwaCazp95zvfYdKkSUyaNIl58+axdetW1q5dyxVXXMENN9zA9ddfz4oVKzj22GM588wz+da3vtWvMYmI9FXuJIv/vR7Wv9zt5kKcya0JCvIiEE2zwjXmKDjn5h6LLF68mOOPP36vdaWlpUyYMIF4PM5zzz3HK6+8wrBhwzjhhBM477zzuPnmm3nllVd2T5wkIpJtaoYKGYZZ0Gw0kM4880wqKiooKipizpw5PPnkkwN6fBGRdOROzaKXGgDA2g3bKIhGmFg5vNey6Zo2bRoPPfTQXuuam5tZvXo1eXl5WKdLond+LCIyGKhmkSIT06ueccYZ7Ny5k/vvvx+ARCLBZz7zGa666iqGDRvG448/zubNm9m1axePPvoos2bNoqSkhG3btvVrHCIiB0LJIkV+tP+vD2VmPPLIIzz44INMnTqVww47jFgsxte+9jUATjzxRC6++GKOPvpoLr74Ympra6moqGDWrFnMmDGDz33uc/0aj4jI/sidZqg07B6Yl3Qikf5rDho/fjy///3vu9w2btw4Hn300X3W//KXv+y344uIHCjVLFLsPn02qWtEiYikUs0iRUE4MK897hQOwDtz1VVXcdVVV2X+QCIiB2jI1yz6cirsUJgEaajMfCgig8uQThaxWIympqa0v0AP9mTh7jQ1NRGLxbIdiogMMUO6GWrcuHHU19fT2NiY9j6btuxiR0GUpoN0Lu5YLMa4ceOyHYaIDDFDOlnk5+czadKkPu1z3W3/oqYsxt1XHZOhqEREDj5Duhlqf9SUxWjY2pLtMEREBhUli06qy2M0bN2V7TBERAYVJYtOqsuKeHNnO7vaEtkORURk0FCy6KS6LDiTSLULEZE9MposzOxsM3vVzOrM7Poutr/dzBaYWdzMLklZf6yZPW1mi83sJTN7XybjTFVdVgTAevVbiIjslrFkYWZR4A7gHGAacJmZTetUbDVwFdD5Qkg7gQ+4+3TgbOC7ZlaeqVhTddQs1ilZiIjslslTZ08E6tx9JYCZPQDMBpZ0FHD3VeG2vUbBuftrKcvrzGwjUAVsyWC8AIzpaIbaomYoEZEOmWyGGgusSXlcH67rEzM7ESgAVnSx7Rozm29m8/sy8K4nsfwoFcMLaGhWzUJEpMOg7uA2s2rgZ8DV7r7PNTjc/S53r3X32qqqqn477piymGoWIiIpMpks1gLjUx6PC9elxcxKgT8CX3T3Z/o5th5VlxVpYJ6ISIpMJovngalmNsnMCoBLgbnp7BiWfwS4390f6q18f6sp1yhuEZFUGUsW7h4HrgXmAUuB37j7YjO7ycwuADCzE8ysHngPcKeZLQ53fy/wduAqM1sY3o7NVKydjSmLsXVXOzvb4gN1SBGRQS2jFxJ098eAxzqt+1LK8vMEzVOd9/s58PNMxtaTmnCsxbotLRw6qjhbYYiIDBqDuoM7WzrGWmhgnohIQMmiCx2juNfpkh8iIoCSRZdGlxUC0LBFNQsREVCy6FJhXpTK4kLWN6tmISICShbdqimPsU41CxERQMmiW2NKNQmSiEgHJYtu1JRrFLeISAcli25Ul8XY1hJne6sG5omIKFl0Q5cqFxHZQ8miGzXlwVgLNUWJiChZdEtzcYuI7KFk0Y3RpTHM0OmzIiIoWXQrPxqhqrhQNQsREZQselSt02dFRAAlix5Vl2oSJBERULLoUXV5MBe3u2c7FBGRrFKy6EFNWRE72hJs08A8EclxShY92DMwT01RIpLblCx6UFMeJAtNgiQiuU7JogcdM+ZpelURyXVKFj0YVVJIxHR9KBERJYse5EUjjCqJsU41CxHJcUoWvaguj6kZSkRynpJFL6rLYurgFpGcp2TRi+qyIhq2tGhgnojkNCWLXlSXxdjVnqB5lwbmiUjuymiyMLOzzexVM6szs+u72P52M1tgZnEzu6TTtivNbHl4uzKTcfak4/RZNUWJSC7LWLIwsyhwB3AOMA24zMymdSq2GrgK+GWnfUcCNwAnAScCN5jZiEzF2pPqck2CJCKSyZrFiUCdu6909zbgAWB2agF3X+XuLwHJTvu+C3jc3Te7+5vA48DZGYy1WzVlml5VRCSTyWIssCblcX24rt/2NbNrzGy+mc1vbGzc70B7UlVSSDRiuj6UiOS0g7qD293vcvdad6+tqqrKyDGiEWN0SaH6LEQkp2UyWawFxqc8Hheuy/S+/a66vEgD80Qkp2UyWTwPTDWzSWZWAFwKzE1z33nAWWY2IuzYPitclxXVZZoxT0RyW8aShbvHgWsJvuSXAr9x98VmdpOZXQBgZieYWT3wHuBOM1sc7rsZ+DJBwnkeuClclxXVZTHWacY8EclheZl8cnd/DHis07ovpSw/T9DE1NW+9wD3ZDK+dFWXFdEaT/LmznZGDi/IdjgiIgPuoO7gHig1GmshIjlOySINYzrGWuj0WRHJUUoWaagpU81CRHKbkkUaKosLyYuYzogSkZylZJGGSMQYXarTZ0UkdylZpKmmPDh9VkQkFylZpKm6rIj1zapZiEhuUrJIU8cobg3ME5FcpGSRpuqyGG3xJE072rIdiojIgFOySFN1eTDWQhcUFJFcpGSRpupwrIU6uUUkFylZpKlaM+aJSA5TskhTxfACCqIRJQsRyUlpJQszm2JmheHyO8zsP82sPLOhDS6RiDG6rFCX/BCRnJRuzeK3QMLMDgXuIpjF7pcZi2qQqi4r0sUERSQnpZsskuFkRhcBt7v754DqzIU1ONWUxWhoVs1CRHJPusmi3cwuA64E/hCuy89MSINXx1zcyaQG5olIbkk3WVwNnAx81d1fN7NJwM8yF9bgVF0Woz3hbNrRmu1QREQGVFrTqrr7EuA/AcxsBFDi7t/IZGCDUcfps+u3tjCqJJblaEREBk66Z0P93cxKzWwksAD4sZndmtnQBp89A/PUyS0iuSXdZqgyd28G5gD3u/tJwDszF9bgVK0Z80QkR6WbLPLMrBp4L3s6uHPOyOEFFOZpYJ6I5J50k8VNwDxghbs/b2aTgeWZC2twMrPdlyoXEckl6XZwPwg8mPJ4JXBxpoIazMaUxWjQxQRFJMek28E9zsweMbON4e23ZjYu08ENRjVlRapZiEjOSbcZ6l5gLlAT3n4frss51eUxNjS3kNDAPBHJIekmiyp3v9fd4+Htp0BVbzuZ2dlm9qqZ1ZnZ9V1sLzSzX4fbnzWzieH6fDO7z8xeNrOlZvaFPrymjBpTVkQ86WzaroF5IpI70k0WTWZ2hZlFw9sVQFNPO5hZFLgDOAeYBlxmZtM6FfsQ8Ka7Hwp8B+gY6PceoNDdjwKOB/6jI5FkW40mQRKRHJRusvggwWmz64EG4BLgql72ORGoc/eV7t4GPADM7lRmNnBfuPwQcIaZGeDAcDPLA4qANqA5zVgzKnUUt4hIrkgrWbj7G+5+gbtXufsod7+Q3s+GGgusSXlcH67rskx4VdutQAVB4thBkJhWA7e4++bOBzCza8xsvpnNb2xsTOelHLDdo7iVLEQkhxzITHnX9VsU+zoRSBB0pk8CPhOO7diLu9/l7rXuXltV1WsXSr8oH5ZPLD+i02dFJKccSLKwXravJZgkqcO4cF2XZcImpzKCvpD3A39y93Z33wj8G6g9gFj7jZkFp882q2YhIrnjQJJFb+eOPg9MNbNJZlYAXEpw+m2quQRzZEDQD/KEuztB09PpAGY2HHgLsOwAYu1XGpgnIrmmxxHcZraNrpOCEXQ8d8vd42Z2LcFlQqLAPe6+2MxuAua7+1zgbuBnZlYHbCZIKBCcRXWvmS0Oj3Wvu7/Uh9eVUdVlRTy1YlO2wxARGTA9Jgt3LzmQJ3f3x4DHOq37UspyC8Fpsp33297V+sGipjzGxm2txBNJ8qIHUjkTETk46JtuP4wpi5FIOo0amCciOULJYj/UhGMtNAmSiOQKJYv9UF0ejLXQwDwRyRVKFvuhYxS3ZswTkVyhZLEfSmN5DCuIqhlKRHKGksV+2DNjnmoWIpIblCz2U025JkESkdyhZLGfxpSqZiEiuUPJYj9VlxexcVsr7YlktkMREck4JYv9VFMWwx02btPAPBEZ+pQs9tOYcF4LXVBQRHKBksV+qikPR3Grk1tEcoCSRet2eOzz0LSiT7t1zJi3Xp3cIpIDlCxam2HRA/DoxyGZSHu3klg+xYV5GpgnIjlByaK0Bs75Bqx5Bp75QZ921cA8EckVShYAx1wKh58Lf/0yNL6a9m7V5UW6mKCI5AQlCwAzOP+7UDAMHv0YJOJp7VZdGlMHt4jkBCWLDiWj4dxbYO0L8NRtae1SXR5j0/ZW2uIamCciQ5uSRaoZF8O02fC3r8OGxb0Wrykrwh02NKt2ISJDm5JFKjM471aIlcEjH4VEe4/Fdw/MU1OUiAxxShadDa+Ed38X1r8E/7ylx6I15R3JQmdEicjQpmTRlSPfDUe9F/51C6xb2G2xMbtnzFPNQkSGNiWL7pz7TRhWGTRHxbu+WGBxYR4lsTzq39w5wMGJiAwsJYvuFI2AC26HxqXw9693W+ykSSN5eMFa1mxWwhCRoUvJoieHnQXHXQH/vg3q53dZ5MYLphMx4/MPvUQy6QMcoIjIwFCy6M27vgYlNUFzVPu+HdnjRgzji+cdydMrm/jFs29kIUARkczLaLIws7PN7FUzqzOz67vYXmhmvw63P2tmE1O2HW1mT5vZYjN72cximYy1W7EymP19aFoOT3ylyyKXnjCet02t5GuPLWN1k5qjRGToyViyMLMocAdwDjANuMzMpnUq9iHgTXc/FPgO8I1w3zzg58BH3X068A6g50EPmTTlNKj9EDx9B7zx1D6bzYxvXHw0eRHjcw8tUnOUiAw5maxZnAjUuftKd28DHgBmdyozG7gvXH4IOMPMDDgLeMndFwG4e5O7p3/98Ew48yYonxBcyrxtxz6ba8qL+H/nT+PZ1zdz/9OrBjw8EZFMymSyGAusSXlcH67rsoy7x4GtQAVwGOBmNs/MFpjZ57s6gJldY2bzzWx+Y2Njv7+AvRQWw4U/gDdfh8dv6LLIe2rH8Y7Dq/jGn15l1aZ9E4qIyMFqsHZw5wGnAJeH9xeZ2RmdC7n7Xe5e6+61VVVVmY9q4ilw0sfg+R/Dyr/vs9nMuHnO0eRFdXaUiAwtmUwWa4HxKY/Hheu6LBP2U5QBTQS1kH+6+yZ33wk8BszMYKzpO+NLMHIK/O5aaGneZ/OYshg3vHs6z63azE+fWjXw8YmIZEAmk8XzwFQzm2RmBcClwNxOZeYCV4bLlwBPuLsD84CjzGxYmEROBZZkMNb0FQyDi34EzWvhz//dZZGLZ47l9CNG8c15y3hdzVEiMgRkLFmEfRDXEnzxLwV+4+6LzewmM7sgLHY3UGFmdcB1wPXhvm8CtxIknIXAAnf/Y6Zi7bPxJ8JbPwkL7oPlf9lns5nx9TlHURCN8LkHF5FQc5SIHOQs+CF/8KutrfX587seZZ0R7S1w16lBU9THn4ai8n2KPLygnut+s4j/Pu9IPvy2yQMXm4hImszsBXev7a3cYO3gHvzyY0Fz1PYN8Mv3weaV+xS56LixvPPI0Xxr3qusaNyehSBFRPqHksWBqDkOLroTNi6BH86CZ34EyT1TrJoZX7toBrH8KJ9Vc5SIHMSULA7U0e+Bjz8Dh8yCP/0X/PQ8aFqxe/Oo0hg3zZ7Oi6u38JN/7Vv7EBE5GChZ9IeysXD5gzD7B8Hc3T+cFVwaJBkMOr/gmBrOmjaabz/+GnUbt2U5WBGRvlOy6C9mcNzl8IlnYPKpMO//wj1nw6blmBlfvegohhdE+cyDLxFPJHt/PhGRQUTJor+V1sBlD8BFd8Gm1+BHp8C/b6NqeB43zZ7BojVb+PG/Xs92lCIifaJkkQlmcMz74BPPwpQz4PEvwd1ncX51M+fMGMN3Hn+N1zaoOUpEDh5KFplUMgYu/QVcfDdsXond+XZuqX6CskLjsw8uUnOUiBw0lCwyzQyOuiSoZRx2FsP/9RUeL/0yLWtf4c5/6uwoETk4KFkMlOJR8N6fwSX3Ut7WwGOFX6T1iW+wbG1TtiMTEemVksVAMoMZc+ATz5E4/Dyui/6G5N3v4qkXX852ZCIiPVKyyIbhlRRedj8rTr2dSck1THr0Am6885es1CVBRGSQUrLIoimnfYC8j/yZklgBn1/3aW757i185Q9L2Lore9ONi4h0Rckiy/LHHkPxtf8kv2YGP8j/DgXPfJfTv/U3fvXcal1LSkQGDSWLwaBkNPkf/CPMuITP5/2aWwvu5IaHF/Du25/k2ZXqABeR7FOyGCzyi+Din8BpX+TUlr/wzNjbsJ2NvO+uZ/jELxawZvPObEcoIjlMyWIwMYNTPw+X3MvIrUv4fewGvvLWCH9dtoEzbv0H3/7zq+xsi2c7ShHJQUoWg9GMOXD1Y0QSbVzxykf495w458wYw+1P1HH6Lf/g0RfXMlRmOBSRg4OSxWA19nj4yBMwchIVcz/AbYc8xW8/+hZGlRby6V8vZM4Pn2LB6jezHaWI5AjNwT3Yte2AR/4Dlv4eZl5J8pxv8dtFG/nmvFdp3NbKtOpS5swcy+xjx1JVUpjtaEXkIJPuHNxKFgeDZBL+9hX417dh4tvgvfezPVrKQ/PX8PCLa3mpfivRiPH2qZXMmTmOM6eNJpYfzXbUInIQULIYihY9AHM/CWXj4P2/gcqpACzfsI2HX1zLoy+upWFrCyWFeZx3dDVzZo6j9pARRCKW5cBFZLBSshiqVj8DD1wOyXZ4z30w5bTdmxJJ59mVTfx2wVr+95UGdrYlGD+yiIuOG8ec48YysXJ4FgMXkcFIyWIoe/MN+NWlsHEJTH0XzPoUHPLW4NTb0M62OPMWr+fhBWt5sm4T7jBzQjlzZo7j/KOrKR9WkMUXICKDhZLFUNe6DZ7+ATx3F+zcFJw99db/hCPfDZG9+yvWb23h0YVreXhBPa9t2E5BNMLpR4zi9CNH8baplVSXFWXpRYhItg2KZGFmZwO3AVHgJ+5+c6fthcD9wPFAE/A+d1+Vsn0CsAS40d1v6elYOZcsOrTvgoW/hKe/D5tXwohJ8NZr4djLg1HhKdydxeuaeXjBWn7/0joat7UCMKVqOKccWskpU6t4y+SRlMTys/FKRCQLsp4szCwKvAacCdQDzwOXufuSlDIfB45294+a2aXARe7+vpTtDwEOPKtk0YtkApb9Af59G6x9AYZVwIn/ASd8GIZX7FPc3Vm2fhtPLt/Ek3WbePb1Jlrak0QjxrHjy8PkUcmx48vJj2o4jshQNRiSxckENYJ3hY+/AODuX08pMy8s87SZ5QHrgSp3dzO7EJgF7AC2K1mkyR3eeAqe+h689ifIK4LjroCTPwEjJ3W7W2s8wYI3tvBkXSNP1jXxcv0Wkg7DC6K8ZXIFp0yt5G1TK5lSVYyZzq4SGSrSTRZ5GYxhLLAm5XE9cFJ3Zdw9bmZbgQozawH+i6BW8tnuDmBm1wDXAEyYMKH/Ij+YmcHEWcFt4zJ46nZ44acw/26YNjvo1xg7c5/dCvOinDylgpOnVPC5d8HWne08tSKodTxZt4m/LtsIwJjSGLMOreSdR47iHYePoqhA4zlEckEmk8WBuBH4jrtv7+lXrLvfBdwFQc1iYEI7iIw6Ai68A07/b3j2RzD/Hlj8SDCwb9an4NB37nUGVaqyYfmcc1Q15xxVDcCazTtTEscGfrugnmEFUU4/YhTnHVWtxCEyxA3KZijgn8D4sFg5kAS+5O7f7+54aoZKQ0szLLgPnvkhNK+FUdPg5GvhqEsgL/1LhcQTSZ59fTN/fLmBP72yns072pQ4RA5Sg6HPIo+gg/sMYC1BB/f73X1xSplPAEeldHDPcff3dnqeG1GfRf+Kt8Ervw3OoNrwChSPgZP+A2qvhqIRfXsqJQ6Rg1rWk0UYxLnAdwlOnb3H3b9qZjcB8919rpnFgJ8BxwGbgUvdfWWn57gRJYvMcIeVfwv6NVY8AfnDYeYH4C0fgxGH9Pnpekoc5x8dJA5ds0pkcBkUyWIgKVkcoPUvw9N3wMsPgidh2oXw1k922Rmeju4SxxmHV3Dq4WOYPraMQ0cV67RckSxTspD9s3UtPHcnzL8XWpsFk/yFAAAQ/0lEQVThkFOCpDH1LIj08Yt952ZY/zKJ9a+wqe4F4g0vU7nrdVYlR/OF9g/zSuQIDhtTzLTqUqbXlDGtppQjq0spLhys512IDD1KFnJgWprhxZ8FlxRprofKw4LO8KPfB/mxvcsm4tC0HDYsDmooGxYHfSHbGvaUGT4KxswgWXk4icVzydu+jhdGX8KPopfzwvp23tzZvrvoxIphu5NHkEhKqSop1PgOkQxQspD+kWiHJb8LRoavfwmGV8EJH4HC4j3JofFVSASXDiGSD1WHw+gZMGYGjJ4eLBeP2vOcrdvgr18OrmtVNg4/71bWj34bS9Y1s3hdM0vWNbOkoZnVm3fu3qWyuIBpNWW7k8e0mlImVgwnqsuvixwQJQvpX+7w+j+DzvC6x4N1YW0hSAhHBfeVh0Femle0XfMc/O5a2PRqUGN519f3ujTJ1l3tLGsIE0h4v3zDNuLJ4DNblB/liOqSIHlUBzWRw0eX9O/ZVzs3B0nt5QeD8Slv/2wwn4jIEKFkIZmzZTXkxfauLeyveGswA+C/boVYKZzzTZhxcbeDBVvjCeo2bt+rBrJ0XTPbWuMARAymVBWnNGEFSWTk8D5ekn3L6qDDf8H90L4Txp0I614M4jr+anjbdVAy5kBfvUjWKVnIwWXDEph7bXARxKnvgvNvTfsXvLuzZvMuljRs3Z1AlqxrZt3Wlt1lxpTGOGxMCSWxPIryowwriFKUH6WoIHU5j1E76zis7m5Gr/4jYDQfdhG7aj9OQc0MRrQ1YP+6BV78BUTzg4s0zvo0FFdl6E0RyTwlCzn4JBPw7J3wxJfBovDOG6D2Q30/Cyu0eUcbSxv21EBWNG5ne2uclrYEO9sT7GxL0BZPAs5JtoyP5s3ltOgidnghv0qczt3xc2lgT7NYZXEBx4wr520V2zl7832MXvU7LK8ITromuObWsJH99EaIDBwlCzl4vbkKfv/pYMDg+LfABbdD1WH9f5xkksSyP8CT3yW67gUSRRVsnnE1DYddwXYrYWdbgl3tCXa1JWhuaWdpwzYW1W9hReN23GGyreP/DvsdpyeeJB4dxqYZH2TEGddRVKqkIQOopRl2NELFlP3aXclCDm7usOhX8KcvBH0Gp34+aPKJ9sPETPFWeOnX8O/vBaf8lh8SjCU57op9JozqSnNLO6/Ub2VR/VYWrdnCttWLeH/Lrzgv+hzNPozfFV3Eiin/hyMOGcsx48uZOqqYvHQHH7btgMZlsHFp0DS3cUnw+ktrgma50nFQNhZKxwaPh1d1278jQ1AiDo1LoX4+rJ0f3De+CuNOgA8/vl9PqWQhQ8P2jfDY52DJozBqOpzxJRheGSSNaGF4XxDc8gr2LEfy9v0SbWmGF+4NLqS4rQHGHBUkoGkXQvTABgJubG5hxctPUzX/Vg59859soZgftZ/PfYmzIH84I4YFSa7jvy2SjDOeBg5NvsFkX80Uf4PJvoaxbCASlmqhgJU2nh0UMcabGMUmCmnf67ht5LEpUkVjpJJNkUqaolU0RarYlDeKzdEqthfVMHb0KKaOLmbqqBIOG12s+dcPJs3rUhLDC8FJFu07gm1FI2BsbZAoJpwEk9+xX4dQspChZdkf4Y+f2XugX49sT+KI5gdX1W1pDv7RJp0Kp3waJp+WmV/laxfgf/saVvc4rQUj+ceoK6iP1DCmZSVjWldS3bKSqrbV5HlwBleCKJsKx7EhNpkNscmsj01mQ2wSmwvG4hYFnGQSEskkRfEtlLWtp6xtI+XtGymPh/ftjYyMb6A80USU5O5QEkRY6FN5In4Mf08ewxI/hIriIqaOKuaw0cUcOrokXC7p+xljAjuagl/6G5cG0xpbBAqKg3FIBcOD5YLU5eF7b0utKbftgHUL99QY1r4QXB0agvFLY46CcWFyGHs8jJzcL59fJQsZelq3wdoFkGgLbvHWYNBgoi0YFLh7uS24sm6iLVzXGixHC4K5yffzeld9tuY5+NtXYeXf96wrGx9cGn7UkcG4lFFHQsXUfUfF769kAratD75kttbDhsV43V+whoUA7CyoYMmwWv6eOJaHmw9nXeue41YML+DQMHFMHV3M5Mpiigoi5EUiRCNGXtTIi0TIixjRiJEfjYT3weO8SCQsY0NvtP2uN4PJxBqXptwvDfoKOuQPC+7bd3b9HF2JFgaJI39Y8EPIE8H68kNSEkNtkCj66zPSiZKFyGCx7sUgaVUdDrGy7MSwfSPU/TUYULniCdj1Jm4R2sfMZG3lKSyK1fLsrvG8unEHyzduZ1tLfL8Ok0ecSrZSFWlme94ItuZVUVSQR2F+hKL8KLH86O77WOq6giixvAix8DTm0lg+lSWFVBYXUFVcyMjhBen3++yvZBLatkHja0FfUUffUeOyvWu0BcXB37LqyCDZjzoiWC6tCX7pJxNBLWH3bdue5dZt3W8rHbun1jCAp2MrWYhI15KJoIlj+eNQ95cgmeFBZ/mUM/BD30nj6Fms3FFIazxJIpkk2bqDvJ0bydu5kfxdjRTs2kjhrkYKWxqJtTZS1LqJotZGitq37HWo1kgRGwsm0JA/nvroeFZHxvI6Y3k9OYbt8Qi72hO0tCfC+2TX8RJ8B48cVkBlcSGVJcH9uFgrE6NNjLWNjIqvZ0RbA8W71pLX0hS8xmR89832epwAD+4tGQ9+zSfjmHc6fl5RkBRGHQlVR4Q1wiOCkwz283TuwUjJQkTSs70xqG3UPR7UPnZtDtreR02H+C7YtiH4BdxZJA+KRwe3kjEpy6ODxLN9A2xaDpteC+63rtmzr0VgxESoPBwqp0LlYXjlVFrLD2VntJStu9rZ/Oab7Ny4kvZNr8OWNyjYVs+wnWspb1tHVXw9Jezd3NPsw1jjVWzyMtrII0GUBEaCKHGiJDwS3NNxHyVOZM+9R2mhgBVewwrGs7WwmpJhhZTG8iktygvuU5eL9l4uieXt1Uy3182MaDS8j3Q02+3bXJdMOm2JJK3xJG3xJK3xYCxQWyJ43HFrTSRpbd+zvrwon3dOG71ff34lCxHpu2QiqGksfxzqn4PC0j2JYK+EMAaKRvbtF3bbDmiqC5p5Nr22J4k01e25ECXAsMrgfuemvffPKwom5So/BMon4OUTaCkZz5v51WyIjGZDe4zG7W1sb4kTsaA2Egm/jM0Mg3C9YbZnXUc5AxLubG+J09zSTvOuONta2mluidO8q333uuaWdna2JQ7kXd7NDPIiRsSMRNJ3X/esr44ZX87vPjFrP2NIL1lo4gAR2SMSDTtWe/3u6LuC4VB9THBLlUwE1+LaXQt5LVjfkRhGTITyCfuMKTGgKLzV9H+0PWpPJNnWKYlsa2mnPekkkkkS4dlre+6DRJD08D58nEi5RSNGQV4kuEUjFOZFKMyL7rVu93Jex/YIBdEowwozPwOlkoWIZFckCiMnBbfDzsp2NGnJj0YYObwgp043Hjq9NCIikjFKFiIi0islCxER6ZWShYiI9ErJQkREeqVkISIivVKyEBGRXilZiIhIr4bM5T7MrBF44wCeohLY1Gupgae4+kZx9Y3i6puhGNch7t7rZW6HTLI4UGY2P53roww0xdU3iqtvFFff5HJcaoYSEZFeKVmIiEivlCz2uCvbAXRDcfWN4uobxdU3ORuX+ixERKRXqlmIiEivlCxERKRXOZUszOxsM3vVzOrM7Pouthea2a/D7c+a2cQBiGm8mf3NzJaY2WIz+1QXZd5hZlvNbGF4+1Km40o59iozezk87j7z1lrge+F79pKZzRyAmA5PeS8WmlmzmX26U5kBec/M7B4z22hmr6SsG2lmj5vZ8vB+RDf7XhmWWW5mVw5AXN8ys2Xh3+kRMyvvZt8e/+YZiOtGM1ub8rc6t5t9e/z/zUBcv06JaZWZLexm30y+X11+P2TlM+buOXEDosAKYDJQACwCpnUq83HgR+HypcCvByCuamBmuFwCvNZFXO8A/pCl920VUNnD9nOB/yWY5fItwLNZ+LuuJxhYNODvGfB2YCbwSsq6bwLXh8vXA9/oYr+RwMrwfkS4PCLDcZ0F5IXL3+gqrnT+5hmI60bgs2n8nXv8/+3vuDpt/zbwpSy8X11+P2TjM5ZLNYsTgTp3X+nubcADwOxOZWYD94XLDwFnmKVM+psB7t7g7gvC5W3AUmBsJo/Zz2YD93vgGaDczKoH8PhnACvc/UBG7+83d/8nsLnT6tTP0X3AhV3s+i7gcXff7O5vAo8DZ2cyLnf/s7vHw4fPAOP663gHElea0vn/zUhc4XfAe4Ff9dfx0tXD98OAf8ZyKVmMBdakPK5n3y/l3WXCf6qtQMWARAeEzV7HAc92sflkM1tkZv9rZtMHKibAgT+b2Qtmdk0X29N5XzPpUrr/J87Wezba3RvC5fXA6C7KZPt9+yBBjbArvf3NM+HasHnsnm6aVLL5fr0N2ODuy7vZPiDvV6fvhwH/jOVSshjUzKwY+C3waXdv7rR5AUEzyzHA7cCjAxjaKe4+EzgH+ISZvX0Aj90jMysALgAe7GJzNt+z3TxoDxhU56eb2ReBOPCLbooM9N/8h8AU4FiggaDJZzC5jJ5rFRl/v3r6fhioz1guJYu1wPiUx+PCdV2WMbM8oAxoynRgZpZP8EH4hbs/3Hm7uze7+/Zw+TEg38wqMx1XeLy14f1G4BGC5oBU6byvmXIOsMDdN3TekM33DNjQ0RQX3m/sokxW3jczuwo4H7g8/JLZRxp/837l7hvcPeHuSeDH3RwvW+9XHjAH+HV3ZTL9fnXz/TDgn7FcShbPA1PNbFL4i/RSYG6nMnOBjjMGLgGe6O4fqr+E7aF3A0vd/dZuyozp6DsxsxMJ/m4DkcSGm1lJxzJBB+krnYrNBT5ggbcAW1Oqx5nW7S++bL1nodTP0ZXA77ooMw84y8xGhM0uZ4XrMsbMzgY+D1zg7ju7KZPO37y/40rt47qom+Ol8/+bCe8Elrl7fVcbM/1+9fD9MPCfsUz04A/WG8GZO68RnFXxxXDdTQT/PAAxgiaNOuA5YPIAxHQKQRXyJWBheDsX+Cjw0bDMtcBigjNAngHeOkDv1+TwmIvC43e8Z6mxGXBH+J6+DNQOUGzDCb78y1LWDfh7RpCsGoB2gjbhDxH0c/0VWA78BRgZlq0FfpKy7wfDz1odcPUAxFVH0Ibd8TnrOPOvBnisp795huP6WfjZeYngS7C6c1zh433+fzMZV7j+px2fqZSyA/l+dff9MOCfMV3uQ0REepVLzVAiIrKflCxERKRXShYiItIrJQsREemVkoWIiPRKyUKkD8wsYXtf8bbfrn5qZhNTr3oqMpjkZTsAkYPMLnc/NttBiAw01SxE+kE4p8E3w3kNnjOzQ8P1E83sifAieX81swnh+tEWzCmxKLy9NXyqqJn9OJy74M9mVpS1FyWSQslCpG+KOjVDvS9l21Z3Pwr4PvDdcN3twH3ufjTBhfu+F67/HvAPDy50OJNg9C/AVOAOd58ObAEuzvDrEUmLRnCL9IGZbXf34i7WrwJOd/eV4YXf1rt7hZltIrh8RXu4vsHdK82sERjn7q0pzzGRYP6BqeHj/wLy3f0rmX9lIj1TzUKk/3g3y33RmrKcQP2KMkgoWYj0n/el3D8dLj9FcIVUgMuBf4XLfwU+BmBmUTMrG6ggRfaHfrWI9E2RmS1Mefwnd+84fXaEmb1EUDu4LFz3SeBeM/sc0AhcHa7/FHCXmX2IoAbxMYKrnooMSuqzEOkHYZ9FrbtvynYsIpmgZigREemVahYiItIr1SxERKRXShYiItIrJQsREemVkoWIiPRKyUJERHr1/wEHEWTP9+71lAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Opt'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[184998    313]\n",
      " [    50 185261]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVYAAAEYCAYAAAAH/d6fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmYFNXVx/Hvb2bYAigIgggCLrgASVAQ1IjiEkVDBBJR3MAlolHfRKNG1BiNgOIejUbjguCGogRBgyCSEKMRFAFRFHVAEMiwIyKbMJz3j7qDxTjT0wzd09B9Pnnqme5TVbduz5jD7Vu37pWZ4ZxzLnXyMl0B55zLNp5YnXMuxTyxOudcinlidc65FPPE6pxzKeaJ1TnnUswTq9shklpKMkkFlTzfJB2Q6npVoh6zJHXJdD1cdvDEmkUkTZK0SlKNUvGhkgaWis2TdGLV1rDyJHWRtDBFZX3v92FmbcxsUirKd84Ta5aQ1BLoDBhwWkYr41yO88SaPfoAk4GhQN+SoKR+wDnA7yV9I+kVSU8DzYFXQuz34dgXJS2WtFrSm5LaxMqpJekeSfPD/rck1SpdCUm/DK3htmVVUtK1kook/U/ShaX21ZB0t6QvJS2R9Ei4bm3gNWDvUN9vJO0tKU9Sf0lzJK2QNELSHrHyjpb0X0lfSVog6fyyfh/h2K0t+FCPP4c6/i+8rhH2dZG0UNLVkpaGz3LB9v+5XFYzM9+yYAMKgcuA9sAmoHFs31BgYKnj5wEnlopdCNQFagB/BmbE9j0ETAKaAvnAUeG4lkSt5ALgglCPA8qpY1dgCdAWqA08F849IOy/DxgD7BHq8Qpwe9jXBVhYqrzfEv1j0izU5W/A8LCvBbAGOAuoBjQA2iXz+wBuDeU2AvYE/gsMiNVjczimGnAqsA6on+n/BnzbebaMV8C3FPwR4eiQTBuG97OBq2L7k0qspfbXC0lvd6JvNuuBH5dxXElivQb4GGiWoMwhwODY+wNLEisgYC2wf2z/kcAX4XVZifUT4ITY+ybh91AAXA+MKqceFSXWOcCpsX0nA/Ni9VgPFMT2LwWOyPR/B77tPFul7uS6nU5f4HUzWx7ePxdi9yVbgKR8YBDQi6iVtiXsakjUGqxJlHDKcy1wq5klusG0N/B+7P382Os9gR8A70vaWi2i1nF5WgCjJG2JxYqBxsA+FdQ3kb1L1W1+iJVYYWabY+/XAXUqeS2XhTyx7uJCP+cZQL6kxSFcA6gn6cdm9gFRq7C00rGzge7AiUStt92BVUTJbTmwAdgf+KCcqpwEjJO02MxGlnNMEVHCK9E89no5UUuwjZktSqK+AAuAC83s7dI7JC0AOpZTj4qmdPsfUdKeFavn/yo4x7mt/ObVrq8HUSutNdAubIcA/yG6oQVRv+Z+pc4rHasLbARWELUcbyvZYWZbiL7G3xtuGuVLOrLUsK5ZRH2oD0kqb1TCCOB8Sa0l/QC4udQ1HgPuk9QIQFJTSSfH6ttA0u6x8h4BBklqEY7fU1L3sO9Z4ERJZ0gqkNRAUrsEv4+44cAfQnkNgT8CzyQ43rltZbovwrcd24BxwD1lxM8AFhN9K2kFzAC+Al4O+7sDX4bYNURfZUcT3fCZT5SU4zeWahHd0FoErAbeDLGW4biCcFwHosR1Sjn17R/q9T+im2Xxa9QkSuhzga+J+lB/Ezt3CFHi/4roq3ke8Dvg01DvOcBtseM7A1NCWQuAviFe1u9jHt/1sdYEHiBqYReF1zXDvi58v69367m++WZmyMwnunbOuVTyrgDnnEsxT6zOOZdinlidcy7FPLE651yK5fQ4VhXUMlWvm+lqOODQQ5pXfJCrEvPnz2P58uWq+Mjk5O/Wwmzz+oTH2Ppl482sa6qumWm5nVir16XGQWdkuhoOeHvKg5muggt+0qlDSsuzzesr/P/ZhhkPNUzpRTMspxOrc64KSJCX6Mnk7OOJ1TmXfsqt2zmeWJ1z6aeUddnuEjyxOufSTN5idc65lBLex+qcc6mlnOsKyK32uXMuM5SXeEumCGlIWGfso1jsBUkzwjZP0owQbylpfWzfI7Fz2kv6UFKhpAcUZlaXtIekCZI+Dz/rh7jCcYWSZko6rKK6emJ1zqVZGG6VaEvOUKI5f7cyszPNrJ2ZtQNGAn+P7Z5Tss/MLo3FHwYuJpo+slWszP7ARDNrBUwM7wFOiR3bL5yfkCdW51x6iagrINGWBDN7E1hZ5iWiVucZRJOUl18VqQmwm5lNtmjO1KeIJouHaI7iYeH1sFLxpywymWh1jiaJruOJ1TmXfhV3BTSUNDW29dvOK3QGlpjZ57HYvpKmS/q3pM4h1hSIr8u2MMQgWtm4KLxeTLR2Wsk5C8o5p0x+88o5l2ZJDbdabmY78iztWWzbWi0CmpvZCkntgZcltUm2MDMzSZVeBcATq3MuvQTkp2+4laQC4BdA+5KYmW0kWsMNM3tf0hyi5dYXAc1ipzcLMYAlkpqYWVH4qr80xBex7SKY8XPK5F0Bzrn0S0EfawInArMttvR6WAgyP7zej+jG09zwVf9rSUeEftk+RGu9AYwhWjae8DMe7xNGBxwBrI51GZTJE6tzLs2UquFWw4F3gIMkLZR0UdjVm+/ftDoGmBmGX70EXGpmJTe+LgMeBwqJFqB8LcQHAz+V9DlRsh4c4mOJFrgsJFpJ+LKK6updAc659EvBk1dmdlY58fPLiI0kGn5V1vFTgbZlxFcAJ5QRN+Dy7amrJ1bnXHql5uv+LsUTq3Mu/XwSFuecSyWf6No551LPuwKccy6FhHcFOOdcavlE1845l3rex+qccynmfazOOZdC8q4A55xLOeV5YnXOuZSJ5rn2rgDnnEsdhS2HeGJ1zqWZvMXqnHOplud9rM45l1reYnXOuVTyPlbnnEstIe8KcM65VMu1roDc+mfEOZcRkhJuSZYxRNJSSR/FYrdIWiRpRthOje27XlKhpE8lnRyLdw2xQkn9Y/F9JU0J8RckVQ/xGuF9YdjfsqK6emJ1zqWXQHlKuCVpKNC1jPh9ZtYubGMBJLUmWmSwTTjnr5Lyw8qtDwGnAK2Bs8KxAHeEsg4AVgElixVeBKwK8fvCcQl5YnXOpZVI3FpNtsVqZm8CKys8MNIdeN7MNprZF0QrrHYMW6GZzTWzb4Hnge5hKezjiVZ0BRgG9IiVNSy8fgk4QRVU2hOrcy7tkkisDSVNjW39tqP4KyTNDF0F9UOsKbAgdszCECsv3gD4ysw2l4pvU1bYvzocXy5PrM659FMFGyw3sw6x7dEkS34Y2B9oBxQB96S45pXiowKcc+ml9D15ZWZLtl5Gegx4NbxdBOwTO7RZiFFOfAVQT1JBaJXGjy8pa6GkAmD3cHy5vMXqnEu7VPSxllNuk9jbnkDJiIExQO9wR39foBXwLvAe0CqMAKhOdINrjJkZ8C/g9HB+X2B0rKy+4fXpwD/D8eXyFqtzLq2UoklYJA0HuhD1xy4Ebga6SGoHGDAPuATAzGZJGgF8DGwGLjez4lDOFcB4IB8YYmazwiWuA56XNBCYDjwR4k8AT0sqJLp51ruiunpizZBHbj6HU45py7KVa+jQ6zYAfnRgU/5yY29q1KjG5uItXHnbC0ydNX/rOe1bN2fSsKvpc/2TjHpjBgADf9Odrp3bADD4sXG89Po0AI49/EBuv6on1avlM/2TBVz6p2cpLt7CbnVqMmRgX/ZpUp+C/Hz+/NREnh4zuYo//a5nw4YNnHjcMXy7cSObizfT8xenc9PNf+Lhhx7kwb/8mblz5rCgaBkNGzYE4JUxo7n15pvIy8ujoKCAO+/5Mz85+ugMf4oMCcOtdpSZnVVG+IkyYiXHDwIGlREfC4wtIz6XaNRA6fgGoNf21NW7AjLk6Vcm0/3yh7aJDbqyB4MefY0jeg9mwMOvMujKHlv35eWJgb/tzhuTZ2+NdT26De0O2YdOvQdzzHl3c2WfE6hbuyaSePzW8+jT/0k69LqNL4tWcu7POwFwyRnHMHvuYjqdOZiTL76fwb/rSbWC3FrorTJq1KjBuAn/5N1pHzBl6gxeHz+OKZMnc+RRP2HsuDdo3qLFNscfd/wJ0bHvz+CRx4Zw2aW/ylDNdw7p6grYWXlizZC3p81h5ep128TMYLfaNQHYvU4tipat3rrvst7H8vLED1i2cs3W2CH77cVb0wopLt7Cug3f8uHnizjpqENoUK82327aTOGXSwH45+TZ9DihXXQNoE7tGgDUrlWDVavXsbl4Szo/alaQRJ06dQDYtGkTmzdtQhLtDj2UFi1bfu/4OnXqbE0Ya9euzcrksT08sbqMufbul7jtyh58/toAbr+qJ3/8S9R3vveeu3Pa8T/m0Rf/s83xMz+LEmmtmtVoUK82x3Y4kGZ71Wf5qm8oKMjnsNbNAeh5YjuaNY6G9z3y/L85eN+9mPv6IKa+eAPX3PUSFfTDu6C4uJhO7dvRfO9GHH/iT+nYqVPC40e/PIoftz2YX3T/GY88OqSKarmTqni4VVZJW2KV1DL+TO8OlNNB0gOpqNPOrl+vzvz+nr/T6pSb+P3dI3n45nMAuOvaX/KH+0d/LwFOnDybcW99zL+GXs2w2y9gyswvKA6tzz79n+TOq3/Bf56+hjVrN1K8JYr/9KhDmPnpQvY76UY69b6d+/r3om5oJbvE8vPzmfL+DArnLWTqe+8y66PE/3l379GTDz6azYiRL3PrLTdVUS13PlI0u1WiLdvs9DevzGwqMDXT9agK53TrxNV3Rk/UjZwwnb/+8WwADmvdnKcGXwBAg3p1OPnoNmzevIVXJs3kzifGc+cT4wEYetv5fB6+/k+Z+QUnXvRnAE444mBatWgEwHmnHcE9T04AYO6C5cxbtIKDWjbe5iaZS6xevXoc2+U4Xn99HG3atq3w+KM7H8MXX8xl+fLlW29u5Zps/LqfSLr/qSiQ9KykTyS9JOkHktpL+rek9yWNLxmHJmmSpDskvSvpM0mdQ7yLpFfD6z0lTZA0S9LjkuZLahhax59Ieizse11SrTR/tpQrWraazu1bAdCl44EUfrkMgEO63cLBP7uZg392M6PemM6Vt7/AK5Nmkpcn9ti9NgBtW+1N21Z788Y70c2tPetH/YHVqxVw9fk/5bGX3gJgweJVdOl4EACN9qjLgS0b88Wi5VX6OXdFy5Yt46uvvgJg/fr1THxjAgcddHC5x88pLNz6DWP6tGls3LiRBg0SPgWZ1XKtjzXdLdaDgIvM7G1JQ4DLiQbxdjezZZLOJBoOcWFJfcyso6Kpv24GTixV3s1Eg3Nvl9SV72afgWgA8FlmdnEYv/ZL4JnSFQrPIEfPIVerk6rPud2G3X4+ndu3omG9OhSOG8CAR8Zy+YDnuOva0ykoyGPjxs1cMXB4wjKqFeTzxpArAVjzzQYuvHHY1q6Aq/qeyCmd25KXJx578T/8+73PgGhI1qN/Opf3RtyABDfeP5oVX61N74fNAouLirj4wr4UFxezxbbwy9PP4NSfdeOhvzzAvffcyZLFizn8sB/RteupPPzo44waNZLnnnmKagXVqFmrFk8/+0JWJpBkpWK41a5E6bpxoWjOwjfNrHl4fzxwA9E4sbnhsHygyMxOkjQJuDEk4cbA22Z2gKQuwDVm1k3SDKBnmK0GSSuBA4E6wAQzaxXi1wHVzGxgojrm/aCR1TjojFR+bFdJq957MNNVcMFPOnXg/fenpiwT1tirlTU7J/Ftkrn3nvq+mXVI1TUzLd0t1tJZew0wy8yOLOf4jeFnMdtft42x18XALtcV4Fw2EpBrjfV097E2l1SSRM8GJgN7lsQkVZPUZjvKexs4I5x7ElA/8eHOucwTeXmJt2yT7sT6KXC5pE+IkuBfiCYxuEPSB8AM4KjtKO9PwElhGFcvYDFRK9g5txPzm1cpYmbzgLJum84Ajinj+C6x18uBluH1JGBS2LUaONnMNodW7+FmtpFo8oW2sfPv3vFP4JxLCeVeV8BOP461lObACEl5wLfAxRmuj3OuAoKs/LqfyC6VWM3sc+DQTNfDObd9PLE651wqeVeAc86lVjTcKrcyqydW51yaZeeQqkQ8sTrn0i7XWqzZN1+Xc27nEvpYE21JFSMNkbQ0Ph2ppLskzZY0U9IoSfVCvKWk9ZJmhO2R2DntJX0oqVDSAwpZX9IeYZKnz8PP+iGucFxhuM5hFdXVE6tzLq1K+lhT8IDAUKBrqdgEoK2Z/Qj4DLg+tm+OmbUL26Wx+MNEQzVbha2kzP7AxDDnyMTwHuCU2LH9wvkJeWJ1zqVdKh5pNbM3iVZJjcdeN7PN4e1koFmiMsI0pbuZ2eSwhPVTQMnict2BYeH1sFLxpywyGainbZfd/v7nTeoTOefcDkiiK6ChpKmxrV8lLnMh8Frs/b6Spof5nzuHWFNgYeyYhSEG0NjMisLrxUDj2DkLyjmnTH7zyjmXXkrq5tXyHZk2UNKNwGbg2RAqApqb2QpJ7YGXt2fCJzMzSZWeU9UTq3MurZTm4VaSzge6ASeEr/eEOUQ2htfvS5pDNHfzIrbtLmgWYgBLJDUxs6LwVX9piC8C9innnDJ5V4BzLu1SMSqg7HLVFfg9cJqZrYvF95SUH17vR3TjaW74qv+1pCPCaIA+wOhw2higb3jdt1S8TxgdcASwOtZlUCZvsTrn0i4V41glDQe6EPXHLiRaqul6oAYwIVxjchgBcAxwq6RNwBbgUjMrufF1GdEIg1pEfbIl/bKDiSZ5ugiYT5j7GRgLnAoUAuuACyqqqydW51x6pWiuADM7q4zwE+UcOxIYWc6+qcSmGY3FVwAnlBE3ovX6kuaJ1TmXVtG0gbnV6+iJ1TmXdjn2RKsnVudc+uXaXAGeWJ1zaSX57FbOOZdyOdZgLT+xStot0Ylm9nXqq+Ocy0Z5OZZZE7VYZwFGdFOvRMl7I1rYzznnEpJ8zautzGyf8vY559z2yLG8mtwjrZJ6S7ohvG4WJjVwzrmkpGg+1l1GhYlV0oPAccB5IbQOeKT8M5xzblvpmitgZ5XMqICjzOwwSdMBzGylpOpprpdzLksIyM/G7JlAMol1k6Q8ohtWSGpANKmBc85VLEu/7ieSTB/rQ0STGewp6U/AW8Adaa2Vcy6reFdAKWb2lKT3gRNDqJeZfZToHOecKyEgP8eGBST75FU+sImoOyC3pqlxzu0w7wooJawlMxzYm2hJguckXZ/4LOeci1TUDZCNOTeZFmsf4NCSZQ8kDQKmA7ens2LOueyRa4+0JvO1vohtE3BBiDnnXFLypIRbMiQNkbRU0kex2B6SJkj6PPysH+KS9ICkQkkzJR0WO6dvOP5zSX1j8faSPgznPBDWxCr3Ggk/b4IPcZ+ke4GVwCxJj0t6DPgQWJ7Ub8I5l/NE9Ehroi1JQ4GupWL9gYlm1gqYGN4DnEK0gGAroB/wMERJkmitrE5AR+DmWKJ8GLg4dl7XCq5RrkRdASX/KswC/hGLT66oUOec2ypF41jN7E1JLUuFuxMtMAgwDJgEXBfiT4X1qiZLqheWtO4CTChZWFDSBKCrpEnAbmY2OcSfAnoQLTRY3jXKlWgSljIX6XLOue2VxtmtGseWol4MNA6vmwILYsctDLFE8YVlxBNdo1wV3ryStD8wCGgN1CyJm9mBFZ3rnHMlXQEVaChpauz9o2b26PZcx8xMkm1n9bZLstdIZlTAUGAgcDdRv8UFhMdbnXMuGUl0BSw3sw6VKHqJpCZmVhS+6i8N8UVAfOrTZiG2iO++1pfEJ4V4szKOT3SNciUzKuAHZjYewMzmmNkfiBKsc85VSIomYUm07YAxQMmd/b7A6Fi8TxgdcASwOnydHw+cJKl+uGl1EjA+7Pta0hFhNECfUmWVdY1yJdNi3RgmYZkj6VKiLF43ifOccw5IzUMAkoYTtTYbSlpIdHd/MDBC0kXAfOCMcPhY4FSgkGiq0wtg6+x8A4D3wnG3ltzIAi4j+oZei+im1WshXt41ypVMYr0KqA38hqivdXfgwiTOc845IDWPtJrZWeXsOqGMYw24vJxyhgBDyohPBdqWEV9R1jUSSWYSlinh5Rq+m+zaOeeSlmMPXiVcpXUUCW5Smdkv0lKjKnToIc15e8qDma6GA+offkWmq+CCjZ9+mdLyJPnsVjGecZxzKZFrs1slekBgYlVWxDmXvXJtrtFk52N1zrlK8YmunXMuDXIsryafWCXVMLON6ayMcy77RJNZ51ZmTWYFgY6SPgQ+D+9/LOkvaa+Zcy5rpGjawF1GMn3KDwDdgBUAZvYBcFw6K+Wcyx4lfayJtmyTTFdAnpnNL9WUL05TfZxzWchHBXzfAkkdAZOUD/wf8Fl6q+WcyyY51sWaVGL9NVF3QHNgCfBGiDnnXIX8yasymNlSoHcV1MU5l6VyLK8mtYLAY5QxZ4CZ9UtLjZxzWSVaQSC3MmsyXQFvxF7XBHqy7ZoxzjmXUI7l1aS6Al6Iv5f0NPBW2mrknMsuYQWBXFKZR1r3JYlVCp1zDpJeTDCrJNPHuorv+ljzgJVA/3RWyjmXXXItsSYctxsW1foxsGfY6pvZfmY2oioq55zb9aXiyStJB0maEdu+lnSlpFskLYrFT42dc72kQkmfSjo5Fu8aYoWS+sfi+0qaEuIvSKpe2c+cMLGGdWPGmllx2HzZa+fc9lHJRCzlbxUxs0/NrJ2ZtQPaEy0QOCrsvq9kn5mNBZDUmmiYaBugK/BXSfnhIaeHiFaabg2cFY4FuCOUdQCwCriosh85mSfNZkg6tLIXcM65PCnhtp1OAOaY2fwEx3QHnjezjWb2BdFqrR3DVmhmc83sW+B5oHv4dn488FI4fxjQY3srVqLcxCqppP/1UOC90HSeJmm6pGmVvaBzLrdEXQGJN6IlrafGtkTj5HsDw2Pvr5A0U9IQSfVDrCnbDgtdGGLlxRsAX5nZ5lLxSkl08+pd4DDgtMoW7pxzIPKosFW63Mw6VFhS1O95GnB9CD0MDCC6wT4AuAe4sPJ1TY1EiVUAZjaniurinMtCIqUPCJwCTDOzJQAlP2HrU6KvhreLgH1i5zULMcqJrwDqSSoIrdb48dstUWLdU9LvyttpZvdW9qLOuRyS2smszyLWDSCpiZkVhbc9gY/C6zHAc5LuBfYGWhF9CxfQStK+RImzN3C2mZmkfwGnE/W79gVGV7aSiRJrPlAnVMQ55yolVYsJSqoN/BS4JBa+U1I7oq6AeSX7zGyWpBHAx8Bm4HIzKw7lXAGMJ8pxQ8xsVijrOuB5SQOB6cATla1rosRaZGa3VrZg55wrkYpJWMxsLdFNpnjsvATHDwIGlREfC4wtIz6XaNTADquwj9U553ZUjk0VkDCxnlBltXDOZS35JCzfMbOVVVkR51z2yq20WrnZrZxzLmk+0bVzzqVBbqVVT6zOubQTeTk2b6AnVudcWonkZnvKJp5YnXNpJ+9jdc65FJLfvHLOuZTyrgDnnEsD7wpwzrkUy7FBAZ5YnXPpFXUF5FZm9cTqnEu7HOsJ8MTqnEs3IW+xOudc6gif3co551JL3hXgnHMp54nV7dQOOqAldevUJT8/n4KCAt6eMpWVK1dy3tlnMn/+PFq0aMkzw0dQv379igtzADxy8zmcckxblq1cQ4detwHwowOb8pcbe1OjRjU2F2/hytteYOqs+XRu34oX7+vHvP+tAGD0P2dw+6PjaNa4Ho8P6EOjBnUxgyEj3+ah4ZO2XuPXvY/lkjM6U7zFGPefj7jx/tHssXttnrvrItq3acEzYyZz1R0vZuLjp12qugIkzQPWAMXAZjPrIGkP4AWgJdGaV2eY2SpFA2fvB04F1gHnm9m0UE5f4A+h2IFmNizE2wNDgVpES7f81sysMnX1xLoLGvfGv2jYsOHW93ffOZgux5/Atb/vz113DubuOwcz6PY7MljDXcvTr0zmkRf+zeMD+myNDbqyB4MefY3X3/6Yk49uzaAre3DyxfcD8Pb0Ofzyt49sU8bm4i30v/fvzJi9kDo/qMF/n7uOiVNmM3vuYo7p0IpuXX5IxzMH8+2mzexZvw4AGzZu4ta/vkrrA/amzf5Nqu4DZ0AKb14dZ2bLY+/7AxPNbLCk/uH9dUTLZLcKWyfgYaBTSMQ3Ax2IFiB8X9IYM1sVjrkYmEKUWLsCr1Wmkrn2pFlWevWV0Zx7Xl8Azj2vL6+MeTnDNdq1vD1tDitXr9smZga71a4JwO51alG0bHXCMhYv/5oZsxcC8M26jcz+YjF771kPgH69OnP3kxP4dtNmAJat+gaAdRu+5b8z5rJh46aUfp6dkZR42wHdgWHh9TCgRyz+lEUmA/UkNQFOBiaY2cqQTCcAXcO+3cxscmilPhUra7t5Yt3FSOLnp5zEUR3b88RjjwKwdMkSmjSJWjx77bUXS5csyWQVs8K1d7/EbVf24PPXBnD7VT3541++W2K+04/2ZcoL/Xn5wV9zyH57fe/c5k32oN1BzXjvo3kAHNCiET85dH/efOoaXn/8t7Rv3byqPsZOQxX8D2goaWps61dGMQa8Lun92P7GZlYUXi8GGofXTYEFsXMXhlii+MIy4pWS0a4ASTeY2W3hdT3gbDP7ayXLGgq8amYvpbCKO52Jk96iadOmLF26lG5df8pBBx+8zX5JOfdcdjr069WZ39/zd16eOINf/vRQHr75HH526YPMmL2Ag069ibXrv+Xko1sz4r5+/LD7d6vE165VneF3/4pr7x7JmrUbACjIz2OP3WtzTJ+76dCmBc/ceSGHdLslQ5+s6gkl08e63Mw6VHDM0Wa2SFIjYIKk2fGdZmaSKtUnmmqZbrHeEHtdD7gsUxXZVTRtGv0j2qhRI07r0ZP33nuXRo0bU1QU/aNdVFTEno0aZbKKWeGcbp14eeIMAEZOmE6HNi0AWLN2A2vXfwvA+Lc+plpBPg3q1QagoCCP4XdfzAuvTWX0Pz/YWtaiJV9tLWvqrPls2WI0DP2sOaGCboBk2wFmtij8XAqMAjoCS8LXeMLPpeHwRcA+sdObhViieLMy4pVSZYlV0suhCT9LUj9Jg4FakmZIehYYDOwf3t8lqY6kiZKmSfpQUvdYWX0kzZT0gaSny7jWAElDJeVX1eerCmvXrmXNmjUEdhvkAAAPuklEQVRbX78x4XXatGnLz7qdxjNPR91Mzzw9jG4/756oGJeEomWr6dy+FQBdOh5I4ZfLAGjcoO7WYzq0aUGexIqv1gLR6IJPv1jMA8/8c5uyXpk0k2MPPxCAA5o3onq1ApaHftZcoQq2Cs+XakuqW/IaOAn4CBgD9A2H9QVK+mzGAH0UOQJYHboMxgMnSaovqX4oZ3zY97WkI8KIgj6xsrZbVXYFXGhmKyXVAt4DjgWuMLN2AJJaAm1j7wuAnmb2taSGwGRJY4DWREMljjKz5eEu31aS7gLqAheUNVQi9M30A9in+a7V17V0yRLOPL0nAJuLN3Nm77M56eSutO9wOOeedQbDnnyC5s1b8MzwERmu6a5l2O3n07l9KxrWq0PhuAEMeGQslw94jruuPZ2Cgjw2btzMFQOHA9DzxEO5uFdnNhcXs2HDJvpc/yQAR7Xbj3O6deLDzxYx+fn+ANz84BjGv/Uxw15+h7/dcg5TX7yBbzcV86s/ftcWmP2PP1G3dk2qVyvg58f9iG6XPcTsuYur/peQRikabtUYGBW6uQqA58xsnKT3gBGSLgLmA2eE48cSDbUqJBpudQFAyEEDiHIQwK1mtjK8vozvhlu9RiVHBACoksO0tv9C0i1Az/C2JdHduTfMrE7Y35Koj7RteF8NuA84BtgCHATsC/QC9jKzG0uVPxQ4FJhiZmV1fH9P+/Yd7O0pU3fgU7lUqX/4FZmuggs2fjqCLeuWpqyj/pAfHmpPvvyvhMcceUD995PoY91lVEmLVVIX4ETgSDNbJ2kSULOC084B9gTam9mmMDi4onPeA9pL2iP2r5BzLsNybRKWqupj3R1YFZLqwcARIb4ptEwheqKibqlzloakehzQIsT/CfSS1ACgVFfAOKK+2n+U9Mc45zIvjeNYd0pV1cc6DrhU0ifAp8DkEH8UmClpmpmdI+ltSR8R9W3cAbwi6UNgKjAbwMxmSRoE/FtSMTAdOL/kQmb2YkiqYySdambrq+gzOufKkY3JM5EqSaxmtpHoEbPSJhE9flZy3Nml9h9ZTnnD+O5pi5LY+bHXQ4Ahlautcy6Vojv/uZVZfa4A51x6ZenX/UQ8sTrn0s4Tq3POpZQvzeKccynnLVbnnEsh4YnVOedSzrsCnHMuxbzF6pxzKZZjedUTq3MuzUTOTb7uidU5l1Z+88o559Igx/KqJ1bnXPp5V4BzzqVYjuVVT6zOufTLsbya8VVanXO5YAdXE5S0j6R/Sfo4LEj62xC/RdKisAjpDEmnxs65XlKhpE8lnRyLdw2xQkn9Y/F9JU0J8RckVa/sx/XE6pxLKwnypIRbEjYDV5tZa6IVSC6X1Drsu8/M2oVtbHRNtQZ6A22ArsBfJeWHlZsfIpofujVwVqycO0JZBwCrgIsq+5k9sTrn0m5Hl782syIzmxZerwE+AZomOKU78LyZbTSzL4hWa+0YtkIzm2tm3wLPA93DktfHAy+F84cBPbbvU37HE6tzLv0qzqwNJU2NbeWutBxWdD4UmBJCV0iaKWmIpPoh1hRYEDttYYiVF28AfGVmm0vFK8UTq3MuzRJ3A4SugOVm1iG2PVpmSVIdYCRwpZl9DTwM7A+0A4qAe6roQyXkowKcc2mV7Nf9CsuJVnQeCTxrZn8HMLMlsf2PAa+Gt4uAfWKnNwsxyomvAOpJKgit1vjx281brM659NvxUQECngA+MbN7Y/EmscN6Ah+F12OA3pJqSNoXaAW8C7wHtAojAKoT3eAaY2YG/As4PZzfFxhdqc+Kt1idc1UgyTv/ifwEOA/4UNKMELuB6K5+O8CAecAlAGY2S9II4GOiEQWXm1kxgKQrgPFAPjDEzGaF8q4Dnpc0EJhOlMgrxROrcy7tdjStmtlb5RQzNsE5g4BBZcTHlnWemc0lGjWwwzyxOufSy5e/ds65dMitzOqJ1TmXVgLyciuvemJ1zqWfdwU451yK+SqtzjmXYt5idc65FJKPCnDOudTzrgDnnEu13Mqrnlidc+nnw62ccy6l5F0BzjmXSsJvXjnnXMp5YnXOuRTzrgDnnEslH8fqnHOplaqlWXYlnlidc2mnHGuyemJ1zqVdjuVVT6zOufTLsbzqidU5l3651hWgaNXX3CRpGTA/0/XYQQ2B5ZmuhAOy52/Rwsz2TFVhksYR/W4SWW5mXVN1zUzL6cSaDSRNNbMOma6H87+F+05epivgnHPZxhOrc86lmCfWXd+jma6A28r/Fg7wPlbnnEs5b7E651yKeWJ1zrkU88TqnHMp5onVOedSzBOrc1VEUvVM18FVDU+sWUq59nD2TkjSD2KvewLXhdf+t8lynlizlJmZpBMkDZDURVLKnv12FZPUBrhW0o9CaH+gCKK/TcYq5qqEJ9YsU9IaktQBuBPYC+gL/EpS00zWLcd8CxwI/FLSfkAd4BvYtsXqrdfs5A8IZCFJhwEPAleb2TuSTgGOA1YBz5jZgoxWMItJUkmLVFIz4GZgBnAYUAw8DDQGNgEzzWxZpurq0sdbrFmiVMtnE9AKOB/AzF4DJgJ7AxdIqlHlFcwBpZLqOUADYDDwY6LE2hLoTfQNoi9QNzM1denmLdYsIqkz0NzMnpXUDngMGG1mA8P+U4AvzGx2JuuZ7SRdBfQCLjGzDyU1B/4AzANGmdknkvLNrDiT9XTp4ysI7OJKWkmSjgSuAX4uaTcze1jSr4CHJNU0sz+ElqtLI0l7A6cCPzezFZIKzOxLSXcBA8IxhWa2KaMVdWnliXUXF5JqZ2AI0Ad4GRgc8u0jkn4DPCppKDDH70inVvzrf5BPdMOwMbAC2BLiy4ArgGqeVLOfJ9ZdkKQmwJlm9ucQagm8ZmbvAO9Img38S9ImM3tC0vFm9nWm6putSvWp/gDYZGYLJI0GukpaZ2bzJJ0H9ATOMbP1mayzqxqeWHdNuwHjJTUxsyJgAXCapFrAxjASYBhwm6QVZvZyRmubpWJJ9Wrgh0ATSZcA44CfAS9KmgR0B3p4Us0dfvNqFyWpJvA3YKWZXSXpqbDrfqARcB4wHTgIuNi7ANIjdLV0J+pXnUw00qYP8DlwLFALmGFmczNWSVflfLjVLiQ+pMrMNgD3AQ0k3WRmfYBFwKXAQOBuYA5Qk9xb1j1tJFWLvS4g6lPtS9R/WgiMAv4BtDOz18zs755Uc4+3WHcxkk4A9iVaLvjl8OjkjcDHsWFVuwE/IRpDea6ZfZixCmcRSXWBs4FngROAesBzRH3cjwNdws3Ej4AvgNPNbGOGqusyyFusu4DYY6qdgCeAFsD1kgaa2SyiFuqhkkpuZhUD+wHneVJNjXCjag3R73Yu0e/8uXCHfznwP6C7pHOB/wL/50k1d/nNq11AaAUdTjTo/DdmNkbS48Dfw43pmyT9kfD3NLO1wEMZrHJWkVQ3JFWIkuhnRE9N1SF6TNiAd4FuRP2q3c1sXgaq6nYSnlh3HZ2A04D/SaphZvPDVHQTJFU3s61T0vmNqtQJ/aiXSVoCrCZKnr2AM4HXJZ1rZp9Kmky0SmtNM1uRuRq7nYH3se6kYk9U7QcsNrN1kn4JXAL8CXjXzDZJagHsY2ZvZbTCWSj2N2hIdGNwNdDEzIrDpNVXA+cS9bkeRzS2eGXmaux2Ft7HupMK/4c+BRhB1J/6D2A00R3n64GjJVUzs/meVFOvVMvfiG5ObQYuAzCzb83sdqIbhD8EfutJ1ZXwroCdlKTWwCDgdOAXQEOir5n3S8ojmtTjdKI+PpdCpZ6ougLYB/gS+CnRE221zOxOSV2BCcCzZral/BJdrvHEuhMpNePRRqJW0kHAGcBZZvaNpKPM7D5JI83Mk2oaxJLqZUR9qWcBHwL1ibpinpTUFugInOpJ1ZXmiXUnUHLXOfTddSYap7qRqC91OXB4SKrHANdJ+pWZfZnJOme7MBb4MKJ/1HoR3fVvTjTH6tlEY1dv8cH/riyeWDMsTN7xD0n3A7OI7iy/T7Q+0pdAdaLlPdYT9a3eEuYHcGlkZl9Luhw4GOhpZseF8cSriJ5o+3N4+s257/HEmmHhbv99QH9gLXCRmf1X0v7AQuBIopmRCoEbzOw1H1JVNcxso6R1QIGkHxI9mDGe6MEAT6quXJ5YdwJmNkrSGmAkcDzRkztfEj0WuY+ZXVNyrCfVKvcl8CpwL9HSNr28G8ZVxMex7kTCgP+7gJvMbLikY4kmU+kGLPWEmhlh4pW9gC1mtijT9XE7P0+sOxlJPweGAf8BNhCtqvpKZmvlnNsenlh3QpJ+AdxKNI/qO/7137ldiyfWnZSkPfxJHud2TZ5YnXMuxXyuAOecSzFPrM45l2KeWJ1zLsU8sTrnXIp5Ys1BkoolzZD0kaQXw3wFlS2ri6RXw+vTJPVPcGy9MGPU9l7jFknXJBsvdcxQSadvx7VahsUAnas0T6y5ab2ZtTOztsC3REtmb6XIdv+3YWZjzGxwgkPqESaKdi6beWJ1/wEOCC21TyU9BXwE7CPpJEnvSJoWWrZ1ACR1lTRb0jSiSbgJ8fMlPRheN5Y0StIHYTuKaLb9/UNr+a5w3LWS3pM0U9KfYmXdKOkzSW8RzUmbkKSLQzkfSBpZqhV+oqSpobxu4fh8SXfFrn3Jjv4inSvhiTWHhYXyTiGaxBmgFfBXM2tDNNPWH4ATzewwYCrwO0k1gceAnwPtiZ6hL8sDwL/N7MdE85rOIprBa05oLV8r6aRwzY5AO6C9pGMktQd6h9ipwOFJfJy/m9nh4XqfABfF9rUM1/gZ8Ej4DBcBq83s8FD+xZL2TeI6zlXIZ7fKTbUkzQiv/wM8QTRz03wzmxziRwCtgbejaUipDrxDND/pF2b2OYCkZ4B+ZVzjeKAPQFgVYbWk+qWOOSls08P7OkSJti4wyszWhWuMSeIztZU0kKi7oQ7R9H4lRoRZ/j+XNDd8hpOAH8X6X3cP1/4siWs5l5An1ty03szaxQMhea6Nh4AJZnZWqeO2OW8HCbjdzP5W6hpXVqKsoUAPM/tA0vlAl9i+0o8XWrj2/5lZPAEjqWUlru3cNrwrwJVnMvATSQcASKot6UBgNtAyTMQN0XpQZZkI/Dqcmy9pd2ANUWu0xHjgwljfbVNJjYA3gR6SakmqS9TtUJG6QFGY4u+cUvt6ScoLdd4P+DRc+9fheCQdKKl2EtdxrkLeYnVlMrNloeU3XFKNEP6DmX0mqR/RcjLriLoS6pZRxG+BRyVdBBQDvw4zdb0dhjO9FvpZDwHeCS3mb4BzzWyapBeAD4ClwHtJVPkmYAqwLPyM1+lLojWrdgMuNbMNkh4n6nudpujiy4Aeyf12nEvMJ2FxzrkU864A55xLMU+szjmXYp5YnXMuxTyxOudcinlidc65FPPE6pxzKeaJ1TnnUuz/AQWlzcJz8FjpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(cm, classes, title='Attack detection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acs = []\n",
    "params_num = []\n",
    "best_n = 10\n",
    "for top_n_features in [3,5,10,20,30,40,50,60,70,80,90,100,115]:\n",
    "    print(\"N \" + str(top_n_features))\n",
    "    fs = [it['feature'] for it in scored[:top_n_features]]\n",
    "    X_train = x_train[fs]\n",
    "    X_opt = x_opt[fs]\n",
    "    X_test = x_test[fs]\n",
    "    \n",
    "    scaler = StandardScaler()\n",
    "    scaler.fit(X_train)\n",
    "    X_train = scaler.transform(X_train)\n",
    "    X_opt = scaler.transform(X_opt)\n",
    "    cp = ModelCheckpoint(monitor='val_loss',\n",
    "                               save_best_only=True,\n",
    "                               verbose=0, filepath='temp')\n",
    "    \n",
    "    model = create_model(top_n_features)\n",
    "    model.compile(loss=\"mean_squared_error\",\n",
    "                    optimizer=\"adam\")\n",
    "\n",
    "    es = EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)\n",
    "    model.fit(X_train, X_train,\n",
    "                    epochs=50,\n",
    "                    validation_data=(X_opt, X_opt),\n",
    "                    verbose=1, callbacks=[es, cp])\n",
    "    \n",
    "    \n",
    "    x_opt_predictions = model.predict(X_opt)\n",
    "    mse = np.mean(np.power(X_opt - x_opt_predictions, 2), axis=1)\n",
    "    mean = mse.mean()\n",
    "    std = mse.std()\n",
    "    \n",
    "        \n",
    "    df_benign = pd.DataFrame(X_test, columns=fs)\n",
    "    df_benign['malicious'] = 0\n",
    "    df_malicious = df_attack.sample(n=df_benign.shape[0], random_state=17)[fs]\n",
    "    df_malicious['malicious'] = 1\n",
    "    df2 = df_benign.append(df_malicious)\n",
    "    X_test = df2.drop(columns=['malicious']).values\n",
    "    X_test_scaled = scaler.transform(X_test)\n",
    "    Y_test = df2['malicious']\n",
    "    \n",
    "    tr = mean + best_n * std\n",
    "    m = AnomalyModel(model , tr, scaler)\n",
    "    Y_pred = m.predict(X_test_scaled)\n",
    "    print('Test-----------------')\n",
    "    print('Accuracy')\n",
    "    acc = accuracy_score(Y_test, Y_pred)\n",
    "    print(acc)\n",
    "    acs.append(acc)\n",
    "    params_num.append(model.count_params())\n",
    "#print(accs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.xlabel('Features')\n",
    "plt.ylabel('Accuracy') \n",
    "plt.ylim(bottom=0.995)\n",
    "plt.xticks([3,10,20,30,40,50,60,70,80,90,100,115])\n",
    "plt.plot([3,5,10,20,30,40,50,60,70,80,90,100,115], acs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.xlabel('Features')\n",
    "plt.ylabel('# of parameters')\n",
    "plt.xlim(3, 115)\n",
    "plt.xticks([3] + list(range(10,109,10)) + [115])\n",
    "\n",
    "plt.plot(range(3, 116), params_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = load_model('anomaly/anomaly115.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot_model(model, to_file='model.png', show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = lime.lime_tabular.LimeTabularExplainer(X_test, feature_names=fs, class_names=classes, discretize_continuous=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df_attack.sample(n=1, random_state=47)[fs].values\n",
    "\n",
    "exp = explainer.explain_instance(test[0], m.scale_predict_classes, num_features=5)\n",
    "exp.show_in_notebook(show_table=True, show_all=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.scale_predict_classes(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.as_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\",\".join([\"{0:.2f}\".format(s) for s in test[0]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
